{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0.合并需要的信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_excel(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\2应用研究\\1语料-论文.xlsx')\n",
    "df2 = pd.read_excel(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\2应用研究\\1语料-专利.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>PY 出版年</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9857</td>\n",
       "      <td>Sliding Backstepping Control Design for Roboti...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>slide backstepping control design robotic mani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15811</td>\n",
       "      <td>Calibration of Embedded Force Sensor for Robot...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>calibration embedded force sensor robotic hand...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1142</td>\n",
       "      <td>Constraint Definition for Gripper Selection an...</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>constraint definition gripper selection grasp ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>866</td>\n",
       "      <td>Energy-saving task assignment for robotic fish...</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>energy-saving task assignment robotic fish sen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13042</td>\n",
       "      <td>An Interaction-Torque Controller for Robotic E...</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>interaction-torque controller robotic exoskele...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      编号                                            TI 文献标题  PY 出版年  \\\n",
       "0   9857  Sliding Backstepping Control Design for Roboti...  2014.0   \n",
       "1  15811  Calibration of Embedded Force Sensor for Robot...  2016.0   \n",
       "2   1142  Constraint Definition for Gripper Selection an...  2022.0   \n",
       "3    866  Energy-saving task assignment for robotic fish...  2015.0   \n",
       "4  13042  An Interaction-Torque Controller for Robotic E...  2012.0   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  slide backstepping control design robotic mani...  \n",
       "1  calibration embedded force sensor robotic hand...  \n",
       "2  constraint definition gripper selection grasp ...  \n",
       "3  energy-saving task assignment robotic fish sen...  \n",
       "4  interaction-torque controller robotic exoskele...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>PN 专利号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>AD 申请详细信息</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>patent76290</td>\n",
       "      <td>WO2020041221-A1</td>\n",
       "      <td>Method for facilitating product-by-process for...</td>\n",
       "      <td>WO2020041221-A1    WOUS047124    20 Aug 2019</td>\n",
       "      <td>method facilitate product-by-process construct...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>patent66777</td>\n",
       "      <td>CN112784676-A</td>\n",
       "      <td>Method for processing image in robot, involves...</td>\n",
       "      <td>CN112784676-A    CN11410238    04 Dec 2020</td>\n",
       "      <td>method process image robot involve analyze hum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>patent55578</td>\n",
       "      <td>CN108759672-A;  CN108759672-B</td>\n",
       "      <td>Industrial robot end position measurement and ...</td>\n",
       "      <td>CN108759672-A    CN10624662    16 Jun 2018;   ...</td>\n",
       "      <td>industrial robot position measurement displace...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>patent31925</td>\n",
       "      <td>CN106289378-A;  CN106289378-B</td>\n",
       "      <td>Laser tracker based industrial robot performan...</td>\n",
       "      <td>CN106289378-A    CN10664737    12 Aug 2016;   ...</td>\n",
       "      <td>laser tracker base industrial robot performanc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>patent67950</td>\n",
       "      <td>CN109664291-A</td>\n",
       "      <td>Robot guiding method, involves performing imag...</td>\n",
       "      <td>CN109664291-A    CN10957526    13 Oct 2017</td>\n",
       "      <td>robot guide method involve perform image proce...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            编号                         PN 专利号  \\\n",
       "0  patent76290                WO2020041221-A1   \n",
       "1  patent66777                  CN112784676-A   \n",
       "2  patent55578  CN108759672-A;  CN108759672-B   \n",
       "3  patent31925  CN106289378-A;  CN106289378-B   \n",
       "4  patent67950                  CN109664291-A   \n",
       "\n",
       "                                             TI 文献标题  \\\n",
       "0  Method for facilitating product-by-process for...   \n",
       "1  Method for processing image in robot, involves...   \n",
       "2  Industrial robot end position measurement and ...   \n",
       "3  Laser tracker based industrial robot performan...   \n",
       "4  Robot guiding method, involves performing imag...   \n",
       "\n",
       "                                           AD 申请详细信息  \\\n",
       "0       WO2020041221-A1    WOUS047124    20 Aug 2019   \n",
       "1         CN112784676-A    CN11410238    04 Dec 2020   \n",
       "2  CN108759672-A    CN10624662    16 Jun 2018;   ...   \n",
       "3  CN106289378-A    CN10664737    12 Aug 2016;   ...   \n",
       "4         CN109664291-A    CN10957526    13 Oct 2017   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  method facilitate product-by-process construct...  \n",
       "1  method process image robot involve analyze hum...  \n",
       "2  industrial robot position measurement displace...  \n",
       "3  laser tracker base industrial robot performanc...  \n",
       "4  robot guide method involve perform image proce...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['编号', 'PN 专利号', ' TI 文献标题', ' AD 申请详细信息', '预处理文本'], dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 19256/19256 [00:00<00:00, 152029.21it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "year_ls = []\n",
    "for i in tqdm(range(0,len(df2[' AD 申请详细信息'].values))):\n",
    "    ii = df2[' AD 申请详细信息'].values[i]\n",
    "    year_ls.append(int(ii.split(';')[0][-4:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2['申请年'] = year_ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df2 = df2.iloc[:,[0,2,-1,-2]]\n",
    "new_df2.columns = ['编号','TI 文献标题','PY 出版年','预处理文本']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df2.to_excel(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\2应用研究\\2计算申请年-专利.xlsx',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>PY 出版年</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9857</td>\n",
       "      <td>Sliding Backstepping Control Design for Roboti...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>slide backstepping control design robotic mani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15811</td>\n",
       "      <td>Calibration of Embedded Force Sensor for Robot...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>calibration embedded force sensor robotic hand...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1142</td>\n",
       "      <td>Constraint Definition for Gripper Selection an...</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>constraint definition gripper selection grasp ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>866</td>\n",
       "      <td>Energy-saving task assignment for robotic fish...</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>energy-saving task assignment robotic fish sen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13042</td>\n",
       "      <td>An Interaction-Torque Controller for Robotic E...</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>interaction-torque controller robotic exoskele...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      编号                                            TI 文献标题  PY 出版年  \\\n",
       "0   9857  Sliding Backstepping Control Design for Roboti...  2014.0   \n",
       "1  15811  Calibration of Embedded Force Sensor for Robot...  2016.0   \n",
       "2   1142  Constraint Definition for Gripper Selection an...  2022.0   \n",
       "3    866  Energy-saving task assignment for robotic fish...  2015.0   \n",
       "4  13042  An Interaction-Torque Controller for Robotic E...  2012.0   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  slide backstepping control design robotic mani...  \n",
       "1  calibration embedded force sensor robotic hand...  \n",
       "2  constraint definition gripper selection grasp ...  \n",
       "3  energy-saving task assignment robotic fish sen...  \n",
       "4  interaction-torque controller robotic exoskele...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df = pd.concat([df1,new_df2],axis=0)\n",
    "all_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df.to_excel(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\2应用研究\\3合并专利-论文.xlsx',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.制作停用词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_excel(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\2应用研究\\3合并专利-论文.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>PY 出版年</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9857</td>\n",
       "      <td>Sliding Backstepping Control Design for Roboti...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>slide backstepping control design robotic mani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15811</td>\n",
       "      <td>Calibration of Embedded Force Sensor for Robot...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>calibration embedded force sensor robotic hand...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1142</td>\n",
       "      <td>Constraint Definition for Gripper Selection an...</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>constraint definition gripper selection grasp ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>866</td>\n",
       "      <td>Energy-saving task assignment for robotic fish...</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>energy-saving task assignment robotic fish sen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13042</td>\n",
       "      <td>An Interaction-Torque Controller for Robotic E...</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>interaction-torque controller robotic exoskele...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      编号                                            TI 文献标题  PY 出版年  \\\n",
       "0   9857  Sliding Backstepping Control Design for Roboti...  2014.0   \n",
       "1  15811  Calibration of Embedded Force Sensor for Robot...  2016.0   \n",
       "2   1142  Constraint Definition for Gripper Selection an...  2022.0   \n",
       "3    866  Energy-saving task assignment for robotic fish...  2015.0   \n",
       "4  13042  An Interaction-Torque Controller for Robotic E...  2012.0   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  slide backstepping control design robotic mani...  \n",
       "1  calibration embedded force sensor robotic hand...  \n",
       "2  constraint definition gripper selection grasp ...  \n",
       "3  energy-saving task assignment robotic fish sen...  \n",
       "4  interaction-torque controller robotic exoskele...  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['编号', 'TI 文献标题', 'PY 出版年', '预处理文本'], dtype='object')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls = []\n",
    "for i in data['预处理文本'].values:\n",
    "    for ii in i.split():\n",
    "        ls.append(ii)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3698277"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "result = Counter(ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_ls, v_ls = [], []\n",
    "for k,v in result.items():\n",
    "    k_ls.append(k)\n",
    "    v_ls.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'词':k_ls, '频次':v_ls})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\2应用研究\\词和词频.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>PY 出版年</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9857</td>\n",
       "      <td>Sliding Backstepping Control Design for Roboti...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>slide backstepping control design robotic mani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15811</td>\n",
       "      <td>Calibration of Embedded Force Sensor for Robot...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>calibration embedded force sensor robotic hand...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1142</td>\n",
       "      <td>Constraint Definition for Gripper Selection an...</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>constraint definition gripper selection grasp ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>866</td>\n",
       "      <td>Energy-saving task assignment for robotic fish...</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>energy-saving task assignment robotic fish sen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13042</td>\n",
       "      <td>An Interaction-Torque Controller for Robotic E...</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>interaction-torque controller robotic exoskele...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      编号                                            TI 文献标题  PY 出版年  \\\n",
       "0   9857  Sliding Backstepping Control Design for Roboti...  2014.0   \n",
       "1  15811  Calibration of Embedded Force Sensor for Robot...  2016.0   \n",
       "2   1142  Constraint Definition for Gripper Selection an...  2022.0   \n",
       "3    866  Energy-saving task assignment for robotic fish...  2015.0   \n",
       "4  13042  An Interaction-Torque Controller for Robotic E...  2012.0   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  slide backstepping control design robotic mani...  \n",
       "1  calibration embedded force sensor robotic hand...  \n",
       "2  constraint definition gripper selection grasp ...  \n",
       "3  energy-saving task assignment robotic fish sen...  \n",
       "4  interaction-torque controller robotic exoskele...  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.nltk预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 去掉纯数字\n",
    "def is_hex(word):\n",
    "    word= word.lower()\n",
    "    for i in range(len(word)):\n",
    "        if not( word[i].startswith('0x') or word[i].isdigit() or word[i] in ['a','b','c','d','e','f']):\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "# def is_not_num(x):\n",
    "#     return not( len(x) > 0 and (x.isnumeric() or x[0].isdigit() or is_hex(x)) )\n",
    "\n",
    "def is_not_num(x):\n",
    "    return not( len(x) > 0 and (x.isnumeric() or x[0].isdigit() ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>PY 出版年</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9857</td>\n",
       "      <td>Sliding Backstepping Control Design for Roboti...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>slide backstepping control design robotic mani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15811</td>\n",
       "      <td>Calibration of Embedded Force Sensor for Robot...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>calibration embedded force sensor robotic hand...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1142</td>\n",
       "      <td>Constraint Definition for Gripper Selection an...</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>constraint definition gripper selection grasp ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>866</td>\n",
       "      <td>Energy-saving task assignment for robotic fish...</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>energy-saving task assignment robotic fish sen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13042</td>\n",
       "      <td>An Interaction-Torque Controller for Robotic E...</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>interaction-torque controller robotic exoskele...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      编号                                            TI 文献标题  PY 出版年  \\\n",
       "0   9857  Sliding Backstepping Control Design for Roboti...  2014.0   \n",
       "1  15811  Calibration of Embedded Force Sensor for Robot...  2016.0   \n",
       "2   1142  Constraint Definition for Gripper Selection an...  2022.0   \n",
       "3    866  Energy-saving task assignment for robotic fish...  2015.0   \n",
       "4  13042  An Interaction-Torque Controller for Robotic E...  2012.0   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  slide backstepping control design robotic mani...  \n",
       "1  calibration embedded force sensor robotic hand...  \n",
       "2  constraint definition gripper selection grasp ...  \n",
       "3  energy-saving task assignment robotic fish sen...  \n",
       "4  interaction-torque controller robotic exoskele...  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = data.copy()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 30174/30174 [00:18<00:00, 1623.72it/s]\n"
     ]
    }
   ],
   "source": [
    "# 分词\n",
    "from nltk import word_tokenize\n",
    "token_ls = []\n",
    "text_ls = []\n",
    "for sentence in tqdm(df['预处理文本']):\n",
    "    token_words = word_tokenize(sentence)\n",
    "    text = ' '.join(token_words)\n",
    "    token_ls.append(token_words)   # list of list 分词结果\n",
    "    text_ls.append(text)           # list 分词列表转化为文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\zb\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import nltk\n",
    "# nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 再小写\n",
    "little_ls = []\n",
    "for words_list in token_ls:\n",
    "    little_words= [x.lower() for x in words_list]\n",
    "    little_ls.append(little_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 30174/30174 [05:30<00:00, 91.35it/s]\n"
     ]
    }
   ],
   "source": [
    "# 词性标注\n",
    "from nltk import pos_tag\n",
    "token_pos_ls = []\n",
    "for token in tqdm(little_ls):\n",
    "    pos_words = pos_tag(token)     #词性标注\n",
    "    token_pos_ls.append(pos_words)\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\zb\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping taggers\\averaged_perceptron_tagger.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import nltk\n",
    "# nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 30174/30174 [00:20<00:00, 1444.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20.891327381134033\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 词形归一化\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "huanyuan_ls = []\n",
    "wordnet_lematizer = WordNetLemmatizer()\n",
    "t1 = time.time()\n",
    "for token_words in tqdm(token_pos_ls):\n",
    "    words_lematizer = []\n",
    "    for word, tag in token_words:\n",
    "        if tag.startswith('NN'):\n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word, pos='n')  # n代表名词\n",
    "        elif tag.startswith('VB'): \n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word, pos='v')   # v代表动词\n",
    "        elif tag.startswith('JJ'): \n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word, pos='a')   # a代表形容词\n",
    "        elif tag.startswith('R'): \n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word, pos='r')   # r代表代词\n",
    "        else: \n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word)\n",
    "        words_lematizer.append(word_lematizer)\n",
    "    huanyuan_ls.append(words_lematizer)  # 得到还原列表\n",
    "#     break\n",
    "t2 = time.time()\n",
    "print(t2-t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\zb\\AppData\\Roaming\\nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import nltk\n",
    "# nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\zb\\AppData\\Roaming\\nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import nltk\n",
    "# nltk.download('omw-1.4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 去停用词\n",
    "with open(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\2应用研究\\英文停用词.txt','r') as f:\n",
    "    stop_ls = f.readlines()\n",
    "stop_ls = [i.strip() for i in stop_ls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_ls =[]\n",
    "for words_ls in huanyuan_ls:\n",
    "    cleaned_words = [word for word in words_ls if word not in stop_ls]\n",
    "    clean_ls.append(cleaned_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████| 30174/30174 [00:01<00:00, 21566.14it/s]\n"
     ]
    }
   ],
   "source": [
    "# 去掉纯数字\n",
    "last_text = []\n",
    "for text in tqdm(clean_ls):\n",
    "    res = filter(is_not_num, text)\n",
    "    res = ' '.join(filter(is_not_num, text))\n",
    "    last_text.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 删除长度小于等于2的单词\n",
    "new_text_ls = []\n",
    "for textls in last_text:\n",
    "    new_text_ls.append(\" \".join([word for word in textls.split() if len(word) > 2]))\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['预处理文本2'] = new_text_ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>PY 出版年</th>\n",
       "      <th>预处理文本</th>\n",
       "      <th>预处理文本2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9857</td>\n",
       "      <td>Sliding Backstepping Control Design for Roboti...</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>slide backstepping control design robotic mani...</td>\n",
       "      <td>slide backstepping control design robotic mani...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15811</td>\n",
       "      <td>Calibration of Embedded Force Sensor for Robot...</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>calibration embedded force sensor robotic hand...</td>\n",
       "      <td>calibration embed force sensor robotic hand ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1142</td>\n",
       "      <td>Constraint Definition for Gripper Selection an...</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>constraint definition gripper selection grasp ...</td>\n",
       "      <td>constraint definition gripper selection grasp ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>866</td>\n",
       "      <td>Energy-saving task assignment for robotic fish...</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>energy-saving task assignment robotic fish sen...</td>\n",
       "      <td>energy-saving task assignment robotic fish sen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13042</td>\n",
       "      <td>An Interaction-Torque Controller for Robotic E...</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>interaction-torque controller robotic exoskele...</td>\n",
       "      <td>controller robotic exoskeleton flexible joint ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30169</th>\n",
       "      <td>patent88453</td>\n",
       "      <td>Multi-sensor fusion simultaneous localization ...</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>multi-sensor fusion simultaneous localization ...</td>\n",
       "      <td>multi-sensor fusion simultaneous localization ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30170</th>\n",
       "      <td>patent88458</td>\n",
       "      <td>Autonomous moving object, such as robotic vacu...</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>autonomous move object robotic vacuum clean de...</td>\n",
       "      <td>autonomous move object robotic vacuum clean de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30171</th>\n",
       "      <td>patent88463</td>\n",
       "      <td>Vision sensor based robot locating device obst...</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>vision sensor base robot locate device obstacl...</td>\n",
       "      <td>vision sensor base robot locate device obstacl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30172</th>\n",
       "      <td>patent88484</td>\n",
       "      <td>Fusion IMU and optical distance measuring sens...</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>fusion imu optical distance measure sensor int...</td>\n",
       "      <td>fusion imu optical distance measure sensor int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30173</th>\n",
       "      <td>patent88521</td>\n",
       "      <td>Collision detection boundary value tuning syst...</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>collision detection boundary tune system multi...</td>\n",
       "      <td>collision detection boundary tune system robot...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30174 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                编号                                            TI 文献标题  PY 出版年  \\\n",
       "0             9857  Sliding Backstepping Control Design for Roboti...  2014.0   \n",
       "1            15811  Calibration of Embedded Force Sensor for Robot...  2016.0   \n",
       "2             1142  Constraint Definition for Gripper Selection an...  2022.0   \n",
       "3              866  Energy-saving task assignment for robotic fish...  2015.0   \n",
       "4            13042  An Interaction-Torque Controller for Robotic E...  2012.0   \n",
       "...            ...                                                ...     ...   \n",
       "30169  patent88453  Multi-sensor fusion simultaneous localization ...  2020.0   \n",
       "30170  patent88458  Autonomous moving object, such as robotic vacu...  2018.0   \n",
       "30171  patent88463  Vision sensor based robot locating device obst...  2019.0   \n",
       "30172  patent88484  Fusion IMU and optical distance measuring sens...  2017.0   \n",
       "30173  patent88521  Collision detection boundary value tuning syst...  2020.0   \n",
       "\n",
       "                                                   预处理文本  \\\n",
       "0      slide backstepping control design robotic mani...   \n",
       "1      calibration embedded force sensor robotic hand...   \n",
       "2      constraint definition gripper selection grasp ...   \n",
       "3      energy-saving task assignment robotic fish sen...   \n",
       "4      interaction-torque controller robotic exoskele...   \n",
       "...                                                  ...   \n",
       "30169  multi-sensor fusion simultaneous localization ...   \n",
       "30170  autonomous move object robotic vacuum clean de...   \n",
       "30171  vision sensor base robot locate device obstacl...   \n",
       "30172  fusion imu optical distance measure sensor int...   \n",
       "30173  collision detection boundary tune system multi...   \n",
       "\n",
       "                                                  预处理文本2  \n",
       "0      slide backstepping control design robotic mani...  \n",
       "1      calibration embed force sensor robotic hand ma...  \n",
       "2      constraint definition gripper selection grasp ...  \n",
       "3      energy-saving task assignment robotic fish sen...  \n",
       "4      controller robotic exoskeleton flexible joint ...  \n",
       "...                                                  ...  \n",
       "30169  multi-sensor fusion simultaneous localization ...  \n",
       "30170  autonomous move object robotic vacuum clean de...  \n",
       "30171  vision sensor base robot locate device obstacl...  \n",
       "30172  fusion imu optical distance measure sensor int...  \n",
       "30173  collision detection boundary tune system robot...  \n",
       "\n",
       "[30174 rows x 5 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\2应用研究\\2新语料.xlsx',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
